"""
@Time: 2021/1/12 下午 4:34
@Author: jinzhuan
@File: tokenize_toolkit.py
@Desc: 
"""
import nltk
from ..base_toolkit import BaseToolkit
import threading


class TokenizeToolkit(BaseToolkit):

    def __init__(self, bert_model=None, model_path=None, vocabulary_path=None, device=None,
                 device_ids=None, max_seq_length=None):
        super().__init__(bert_model, model_path, vocabulary_path, device, device_ids, max_seq_length)

    _instance_lock = threading.Lock()

    def __new__(cls, *args, **kwargs):
        if not hasattr(TokenizeToolkit, "_instance"):
            with TokenizeToolkit._instance_lock:
                if not hasattr(TokenizeToolkit, "_instance"):
                    TokenizeToolkit._instance = object.__new__(cls)
        return TokenizeToolkit._instance

    def run(self, sentence):
        words = nltk.word_tokenize(sentence)
        return words
